package net.jxtz.jxtzos.spider.process;


import lombok.extern.slf4j.Slf4j;
import net.jxtz.jxtzos.entity.spider.SpiderBeike;
import net.jxtz.jxtzos.entity.spider.SpiderLogServe;
import net.jxtz.jxtzos.service.spider.ISpiderBeikeService;
import net.jxtz.jxtzos.spider.BaseSpiderProcessor;
import net.jxtz.jxtzos.spider.SpiderDefinition;
import net.jxtz.jxtzos.spider.SpiderFactory;
import net.jxtz.jxtzos.spider.pipeline.BeiKePipline;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Selectable;

import java.io.IOException;
import java.math.BigDecimal;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * 抓取贝壳的逻辑
 * @author lmh
 * @version 1.0
 * @date 2021-09-10 10:01
 */
@Slf4j
@Component
public class BeiKeProcessor implements PageProcessor, BaseSpiderProcessor {

    private final Site site = Site.me().setRetryTimes(50).setRetrySleepTime(1000).setCharset("utf8").setTimeOut(100000);

    private final AtomicInteger nanchangPage = new AtomicInteger(2);
    private final AtomicInteger jiujiangPage = new AtomicInteger(2);
    private final AtomicInteger shangraoPage = new AtomicInteger(2);

    private Integer count = 0;

    public Integer getCount() {
        return count;
    }

    private Integer lastTime;

    public Integer getLastTime() {
        return lastTime;
    }

    public static final String SPIDER_NAME = "beiKe";

    private Long spiderLogId = null;

    public BeiKeProcessor() {
        SpiderFactory instance = SpiderFactory.getInstance();
        instance.putSpiderProcess(SPIDER_NAME, this);
        SpiderDefinition beike = new SpiderDefinition("贝壳", SPIDER_NAME);
        beike.setImgUrl("https://gimg2.baidu.com/image_search/src=http%3A%2F%2Fp1.pstatp.com%2Faweme%2F720x720%2F7bef0000cb1a82066d74.jpeg&refer=http%3A%2F%2Fp1.pstatp.com&app=2002&size=f9999,10000&q=a80&n=0&g=0n&fmt=jpeg?sec=1634455051&t=2d854649d813ed7d27a2f9825c9a607a");
        instance.putSpiderDefinition(SPIDER_NAME, beike);
    }

    @Autowired
    private BeiKePipline beiKePipline;

    @Autowired
    private ISpiderBeikeService service;

    @Override
    public void process(Page page) {
        // 获取页面详情所有超链接
        List<Selectable> list = page.getHtml().css("div.info div.title a").nodes();
        if (list.size() == 0) {
            Selectable url = page.getUrl();
            String s8 = url.toString();
            log.info("当前地址：" + s8);
            // 如果为空，表示详情页，并开始解析详情页
            String s = page.getHtml().toString(); // 最大页面
            Elements main = Jsoup.parse(s).getElementsByClass("main");
            String text = main.text();
            String title = text.substring(0, text.lastIndexOf("米") + 1); // 标题
            String date = text.substring(text.lastIndexOf("米") + 1, text.indexOf("成")); // 成交日期
            Elements dealTotalPrice = Jsoup.parse(s).getElementsByClass("dealTotalPrice");
            String price = dealTotalPrice.text().substring(0, dealTotalPrice.text().indexOf("万")); // 总价
            String price1 = Jsoup.parse(s).getElementsByClass("price").html();
            Elements b = Jsoup.parse(price1).getElementsByTag("b");
            String unitPrice = b.text(); // 单价

            Selectable selectable = page.getHtml().css("div.content").nodes().get(1);
            String base = selectable.toString(); // 基本信息
            Element li = Jsoup.parse(base).getElementsByTag("li").first();
            String s1 = li.text();
            String houseType = s1.substring(4, s1.length()); // 户型

            Elements li1 = Jsoup.parse(base).getElementsByTag("li").eq(1);
            String text1 = li1.text();
            String s2 = text1.substring(4, text1.length());

            String s3 = s2.replaceAll("\\(|\\)", "").replaceAll(" ", "");
            String floor = s3.substring(0, s3.indexOf("共")); // 所在楼层

            String s4 = s3.substring(s3.indexOf("共"), s3.length());
            String allFloor = s4.replaceAll("[^0-9]", ""); // 总楼层，利用正则抽取出数字

            Elements li2 = Jsoup.parse(base).getElementsByTag("li").eq(2);
            String text2 = li2.text();
            String s5 = text2.substring(4, text2.length());
            String area = s5.substring(0, s5.indexOf("㎡")); // 面积

            Elements li3 = Jsoup.parse(base).getElementsByTag("li").eq(8);
            String text3 = li3.text();
            String renovation = text3.substring(4, text3.length()); // 装修


            Elements li4 = Jsoup.parse(base).getElementsByTag("li").eq(6);
            String text4 = li4.text();
            String substring = text4.substring(4, text4.length());
            String orientation = substring.replaceAll(" ", ""); // 朝向

//            // 小区名称
//            Document document1 = page.getHtml().getDocument();
//            log.info(document1.toString());
//            Element aroundInfo = document1.getElementsByClass("aroundInfo").get(0);
//            Element areaName = aroundInfo.getElementsByClass("areaName").get(0);
//            Element info = areaName.getElementsByClass("info").get(0);
//            String a = info.getElementsByTag("a").get(0).text();

            // 获取建成年代
            String dateOfBuild = "";

            Document document = page.getHtml().getDocument();
            // 获取到对应的script标签
            Elements script = document.getElementsByTag("script").eq(13);
            String text5 = script.html();
            // 截取子字符串
            String substring1 = text5.substring(text5.indexOf("resblockId"), text5.indexOf("fbExpoId"));
            String substring2 = substring1.substring(substring1.indexOf("'") + 1, substring1.indexOf(","));
            // 获取到请求地址的参数 hid rid
            String rid = substring2.replaceAll("'", ""); // rid

            String s6 = page.getUrl().toString();
            String hid = s6.substring(s6.lastIndexOf("/") + 1, s6.lastIndexOf(".")); // hid

            // 发送请求获取到响应体
            String requestUrl = "https://nc.ke.com/chengjiao/resblock?hid=" + hid + "&rid=" + rid + "";
            CloseableHttpClient httpClient = HttpClients.createDefault();
            // 发送get请求
            HttpGet httpGet = new HttpGet(requestUrl);
            CloseableHttpResponse response = null;
            try {
                response = httpClient.execute(httpGet);
                if (response.getStatusLine().getStatusCode() == 200) {
                    log.info(requestUrl + "，请求成功");
                    // 获取到响应体
                    HttpEntity entity = response.getEntity();
                    // 响应体流转字符串
                    String s7 = EntityUtils.toString(entity);
                    // 截取字符串
                    String substring3 = s7.substring(s7.indexOf("buildYear"), s7.indexOf("buildNum"));
                    dateOfBuild = substring3.substring(substring3.indexOf(":") + 2, substring3.indexOf(",") - 7);
                }
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                // 关闭流
                if (httpClient != null) {
                    try {
                        httpClient.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }

                if (response != null) {
                    try {
                        response.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
            }

            if (s8.contains("nc")) {
                s8 = "南昌";
            } else if (s8.contains("sr")) {
                s8 = "上饶";
            } else {
                s8 = "九江";
            }

            // 封装进对象 TODO
            SpiderBeike spiderBeike = new SpiderBeike();
            spiderBeike.setTitle(title);
//            spiderBeike.setAreaCity(a);
            spiderBeike.setDate(date);
            spiderBeike.setPrice(BigDecimal.valueOf(Double.parseDouble(price)));
            spiderBeike.setUnitPrice(Integer.valueOf(unitPrice));
            spiderBeike.setHouseType(houseType);
            spiderBeike.setFloor(floor);
            spiderBeike.setAllFloor(allFloor);
            spiderBeike.setArea(BigDecimal.valueOf(Double.parseDouble(area)));
            spiderBeike.setRenovation(renovation);
            spiderBeike.setOrientation(orientation);
            spiderBeike.setDateOfBuild(Integer.valueOf(dateOfBuild));
            spiderBeike.setPageUrl(page.getRequest().getUrl());
            spiderBeike.setCity(s8);
            if (spiderLogId != null) {
                spiderBeike.setSpiderLogId(spiderLogId);
            }

            page.putField("beike", spiderBeike);
        } else {
            // 不为空，表示列表页，解析出详情页url
            for (Selectable selectable : list) {
                // 获取到详情页url
                String url = selectable.links().toString();
                // 把url地址放进任务队列中
                page.addTargetRequest(url);
            }

            String url = page.getRequest().getUrl();
            String city = url.substring(8, url.indexOf("."));
            switch (city){
                case "nc":
                    // 获取下一页url
                    List<Selectable> nodes = page.getHtml().css("div.pagination_group_a a").nodes();
                    Selectable selectable = nodes.get(0).links();
                    String next = selectable.toString() + "pg" + nanchangPage.getAndIncrement();

                    // 将下一页url放入到队列中
                    page.addTargetRequest(next);
                    break;
                case "sr":
                    // 获取下一页url
                    List<Selectable> srnodes = page.getHtml().css("div.pagination_group_a a").nodes();
                    Selectable srselectable = srnodes.get(0).links();
                    String srnext = srselectable.toString() + "pg" + shangraoPage.getAndIncrement();

                    // 将下一页url放入到队列中
                    page.addTargetRequest(srnext);
                    break;
                case "jiujiang":
                    List<Selectable> jiujiang = page.getHtml().css("div.pagination_group_a a").nodes();
                    Selectable jiujiangSelect = jiujiang.get(0).links();
                    String jiujiangNext = jiujiangSelect.toString() + "pg" + jiujiangPage.getAndIncrement();
                    page.addTargetRequest(jiujiangNext);
                    break;
                default:

            }


        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    @Override
    public void start(SpiderLogServe spiderLogServe, SpiderDefinition spiderDefinition) {
        spiderLogId = spiderLogServe.getId();
        long startTime = System.currentTimeMillis() / 1000;
        int startCount = service.count();
        log.info("开始爬取");

        String nanchang = "https://nc.ke.com/chengjiao/";
        String jiujiang = "https://jiujiang.ke.com/chengjiao/";
        String shangrao = "https://sr.ke.com/chengjiao/";
        Spider.create(new BeiKeProcessor())
                .addUrl(nanchang, jiujiang, shangrao)
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(10000)))
                .addPipeline(beiKePipline)
                .thread(3)
                .run();

        long endTime = System.currentTimeMillis() / 1000;
        long time = endTime - startTime;
        int endCount = service.count();
        count = endCount - startCount;
        lastTime = Long.valueOf(endTime).intValue();
        spiderLogServe.setCount(count);
        log.info("爬取完毕，用时：" + time + "s，更新数量：" + count);
    }
}
